#!/bin/bash
# created by kumiler on 2022/11/18 18:30

function task_job() {
jdbcUrl="$1"
db="$2"
mysqlTable="$3"
hiveTable="$4"
column="${5}"
sub="${6}"
threadNumber="${7}"
typed="${8}"
now_dt="{{ execution_date | cst_ds }}"
next_dt="{{ execution_date | date_add(1) | cst_ds }}"
env='{{ var.value.env_sync }}'
username='{{ var.json.mysql_spmibill.username }}'
password='{{ var.json.mysql_spmibill.password }}'
table="spmi_composite_bill"


json='{
"reader":{
"connect":{
"url":"'"${jdbcUrl}"'",
"username":"'"${username}"'",
"password":"'"${password}"'",
"driver":"com.mysql.cj.jdbc.Driver"
},
"dbtype":"mysql",
"tableName":"'"${db}.${mysqlTable}"'",
"subTableList":"'"${sub}"'",
"where":"'"${column} between str_to_date('${now_dt} 00:00:00', '%Y-%m-%d %H:%i:%s') and str_to_date('${next_dt} 00:00:00', '%Y-%m-%d %H:%i:%s')"'",
"query":"'""'",
"splitColumn":"'"${column}"'",
"equalitySectioning":0,
"containsnull":0,
"fetchsize":"1024",
"threadNumber":0
},
"channel":{
"filterAbnormalCharacter":0
},
"writer":{
"dag":"jms_ods",
"task_id":"spmi_ods__spmi_composite_bill",
"dbtype":"hive",
"tableName":"'"${hiveTable}"'",
"database":"spmi_ods",
"writeMode": "'"${typed}"'",
"partitionColumn":"dt",
"partitionValue":"'"${now_dt}"'"},
"settting":{
"env":"'"${env}"'"}
}'

/usr/hdp/3.1.4.0-315/spark2/bin/spark-submit \
--master yarn \
--deploy-mode cluster \
--driver-memory 32G \
--driver-cores 4 \
--executor-memory 18G \
--executor-cores 4 --queue pro  \
--num-executors 20 \
--conf spark.driver.maxResultSize=6g  \
--class com.yunlu.bigdata.jobs.synchrotool.DataSynchDriver \
hdfs:///scheduler/jms/spark/sync/mysql/spark_sync.jar  "$json"

echo ${json} >> /jms_ods/spmi_composite_bill.log
echo "任务执行成功: ${dt} ${hiveTable} ${db} ${sub}"
}
#jdbcUrl='{{ var.json.mysql_spmi.db2 }}'

jdbcUrl0='{{ var.json.mysql_spmibill.yl_spmibill_composite_0 }}'
jdbcUrl1='{{ var.json.mysql_spmibill.yl_spmibill_composite_1 }}'
jdbcUrl2='{{ var.json.mysql_spmibill.yl_spmibill_composite_2 }}'
jdbcUrl3='{{ var.json.mysql_spmibill.yl_spmibill_composite_3 }}'


jdbcUrl4='{{ var.json.mysql_spmibill.yl_spmibill_composite_4 }}'
jdbcUrl5='{{ var.json.mysql_spmibill.yl_spmibill_composite_5 }}'
jdbcUrl6='{{ var.json.mysql_spmibill.yl_spmibill_composite_6 }}'
jdbcUrl7='{{ var.json.mysql_spmibill.yl_spmibill_composite_7 }}'


jdbcUrl8='{{ var.json.mysql_spmibill.yl_spmibill_composite_8 }}'
jdbcUrl9='{{ var.json.mysql_spmibill.yl_spmibill_composite_9 }}'
jdbcUrl10='{{ var.json.mysql_spmibill.yl_spmibill_composite_10 }}'
jdbcUrl11='{{ var.json.mysql_spmibill.yl_spmibill_composite_11 }}'

jdbcUrl12='{{ var.json.mysql_spmibill.yl_spmibill_composite_12 }}'
jdbcUrl13='{{ var.json.mysql_spmibill.yl_spmibill_composite_13 }}'
jdbcUrl14='{{ var.json.mysql_spmibill.yl_spmibill_composite_14 }}'
jdbcUrl15='{{ var.json.mysql_spmibill.yl_spmibill_composite_15 }}'


mTab="spmi_composite_bill"
hTab="spmi_composite_bill"
col='update_time'

# 2024-01-10 實例下線 by kumiler
#task_job  ${jdbcUrl}  yl_spmibill_2 ${mTab} ${hTab} ${col} "[0,364]"  8  overwrite
# 增加实例20221121
task_job  ${jdbcUrl0}  yl_spmibill_composite_0 ${mTab} ${hTab} ${col} '[0,127]' 8 overwrite
task_job  ${jdbcUrl1}  yl_spmibill_composite_1 ${mTab} ${hTab} ${col} '[0,127]' 3 append
task_job  ${jdbcUrl2}  yl_spmibill_composite_2 ${mTab} ${hTab} ${col} '[0,127]' 3 append
task_job  ${jdbcUrl3}  yl_spmibill_composite_3 ${mTab} ${hTab} ${col} '[0,127]' 3 append
wait

task_job  ${jdbcUrl4}  yl_spmibill_composite_4 ${mTab} ${hTab} ${col} '[0,127]' 3 append
task_job  ${jdbcUrl5}  yl_spmibill_composite_5 ${mTab} ${hTab} ${col} '[0,127]' 3 append
task_job  ${jdbcUrl6}  yl_spmibill_composite_6 ${mTab} ${hTab} ${col} '[0,127]' 3 append
task_job  ${jdbcUrl7}  yl_spmibill_composite_7 ${mTab} ${hTab} ${col} '[0,127]' 3 append
wait

task_job  ${jdbcUrl8}  yl_spmibill_composite_8 ${mTab} ${hTab} ${col} '[0,127]' 3 append
task_job  ${jdbcUrl9}  yl_spmibill_composite_9 ${mTab} ${hTab} ${col} '[0,127]' 3 append
task_job  ${jdbcUrl10}  yl_spmibill_composite_10 ${mTab} ${hTab} ${col} '[0,127]' 3 append
task_job  ${jdbcUrl11}  yl_spmibill_composite_11 ${mTab} ${hTab} ${col} '[0,127]' 3 append
wait

task_job  ${jdbcUrl12}  yl_spmibill_composite_12 ${mTab} ${hTab} ${col} '[0,127]' 3 append
task_job  ${jdbcUrl13}  yl_spmibill_composite_13 ${mTab} ${hTab} ${col} '[0,127]' 3 append
task_job  ${jdbcUrl14}  yl_spmibill_composite_14 ${mTab} ${hTab} ${col} '[0,127]' 3 append
task_job  ${jdbcUrl15}  yl_spmibill_composite_15 ${mTab} ${hTab} ${col} '[0,127]' 3 append
wait

echo "${hTab} 表所有任务执行成功! "
